{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "v0FebFOcQGkQ"
   },
   "source": [
    "https://github.com/lucidrains/reformer-pytorch/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 194,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/kaggle/input/reformerwritemodel/model/scheduler.pt\n",
      "/kaggle/input/reformerwritemodel/model/model.pt\n",
      "/kaggle/input/reformerwritemodel/model/optimizer.pt\n",
      "/kaggle/input/gpt2writedata/data.json\n",
      "/kaggle/input/gpt2writedata/article.db\n",
      "/kaggle/input/gpt2writedata/train.json\n",
      "/kaggle/input/gpt2writedata/train.txt\n",
      "/kaggle/input/gpt2writedata/train_clear.txt\n",
      "/kaggle/input/gpt2writedata/train_clear_web_mini.txt\n"
     ]
    }
   ],
   "source": [
    "# This Python 3 environment comes with many helpful analytics libraries installed\n",
    "# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n",
    "# For example, here's several helpful packages to load in \n",
    "\n",
    "import numpy as np # linear algebra\n",
    "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
    "\n",
    "# Input data files are available in the \"../input/\" directory.\n",
    "# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n",
    "\n",
    "import os\n",
    "for dirname, _, filenames in os.walk('/kaggle/input'):\n",
    "    for filename in filenames:\n",
    "        print(os.path.join(dirname, filename))\n",
    "\n",
    "# Any results you write to the current directory are saved as output."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 231,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 469
    },
    "colab_type": "code",
    "id": "rNoUIwU2NU-V",
    "outputId": "a07c217c-9d05-48e9-9e00-313a9abf3c6c"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting reformer-pytorch==0.15.1\n",
      "  Downloading reformer_pytorch-0.15.1.tar.gz (14 kB)\n",
      "Requirement already satisfied: torch in /opt/conda/lib/python3.6/site-packages (from reformer-pytorch==0.15.1) (1.4.0)\n",
      "Building wheels for collected packages: reformer-pytorch\n",
      "  Building wheel for reformer-pytorch (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25h  Created wheel for reformer-pytorch: filename=reformer_pytorch-0.15.1-py3-none-any.whl size=12895 sha256=49f6bbf4ed85de83e007edf380c693330c90f0da255b3c5bccbade4af2c3b7a4\n",
      "  Stored in directory: /root/.cache/pip/wheels/a6/11/e3/fc125628c729132c606de958cdc714e68dd039384c4b4c7fcb\n",
      "Successfully built reformer-pytorch\n",
      "Installing collected packages: reformer-pytorch\n",
      "  Attempting uninstall: reformer-pytorch\n",
      "    Found existing installation: reformer-pytorch 0.14.0\n",
      "    Uninstalling reformer-pytorch-0.14.0:\n",
      "      Successfully uninstalled reformer-pytorch-0.14.0\n",
      "Successfully installed reformer-pytorch-0.15.1\n",
      "Requirement already satisfied: transformers==2.4.1 in /opt/conda/lib/python3.6/site-packages (2.4.1)\n",
      "Requirement already satisfied: boto3 in /opt/conda/lib/python3.6/site-packages (from transformers==2.4.1) (1.11.15)\n",
      "Requirement already satisfied: tokenizers==0.0.11 in /opt/conda/lib/python3.6/site-packages (from transformers==2.4.1) (0.0.11)\n",
      "Requirement already satisfied: regex!=2019.12.17 in /opt/conda/lib/python3.6/site-packages (from transformers==2.4.1) (2020.1.8)\n",
      "Requirement already satisfied: numpy in /opt/conda/lib/python3.6/site-packages (from transformers==2.4.1) (1.18.1)\n",
      "Requirement already satisfied: requests in /opt/conda/lib/python3.6/site-packages (from transformers==2.4.1) (2.22.0)\n",
      "Requirement already satisfied: tqdm>=4.27 in /opt/conda/lib/python3.6/site-packages (from transformers==2.4.1) (4.42.1)\n",
      "Requirement already satisfied: sacremoses in /opt/conda/lib/python3.6/site-packages (from transformers==2.4.1) (0.0.38)\n",
      "Requirement already satisfied: sentencepiece in /opt/conda/lib/python3.6/site-packages (from transformers==2.4.1) (0.1.85)\n",
      "Requirement already satisfied: filelock in /opt/conda/lib/python3.6/site-packages (from transformers==2.4.1) (3.0.12)\n",
      "Requirement already satisfied: jmespath<1.0.0,>=0.7.1 in /opt/conda/lib/python3.6/site-packages (from boto3->transformers==2.4.1) (0.9.4)\n",
      "Requirement already satisfied: s3transfer<0.4.0,>=0.3.0 in /opt/conda/lib/python3.6/site-packages (from boto3->transformers==2.4.1) (0.3.3)\n",
      "Requirement already satisfied: botocore<1.15.0,>=1.14.15 in /opt/conda/lib/python3.6/site-packages (from boto3->transformers==2.4.1) (1.14.15)\n",
      "Requirement already satisfied: chardet<3.1.0,>=3.0.2 in /opt/conda/lib/python3.6/site-packages (from requests->transformers==2.4.1) (3.0.4)\n",
      "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /opt/conda/lib/python3.6/site-packages (from requests->transformers==2.4.1) (1.25.8)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /opt/conda/lib/python3.6/site-packages (from requests->transformers==2.4.1) (2019.11.28)\n",
      "Requirement already satisfied: idna<2.9,>=2.5 in /opt/conda/lib/python3.6/site-packages (from requests->transformers==2.4.1) (2.8)\n",
      "Requirement already satisfied: click in /opt/conda/lib/python3.6/site-packages (from sacremoses->transformers==2.4.1) (7.0)\n",
      "Requirement already satisfied: six in /opt/conda/lib/python3.6/site-packages (from sacremoses->transformers==2.4.1) (1.14.0)\n",
      "Requirement already satisfied: joblib in /opt/conda/lib/python3.6/site-packages (from sacremoses->transformers==2.4.1) (0.14.1)\n",
      "Requirement already satisfied: python-dateutil<3.0.0,>=2.1 in /opt/conda/lib/python3.6/site-packages (from botocore<1.15.0,>=1.14.15->boto3->transformers==2.4.1) (2.8.1)\n",
      "Requirement already satisfied: docutils<0.16,>=0.10 in /opt/conda/lib/python3.6/site-packages (from botocore<1.15.0,>=1.14.15->boto3->transformers==2.4.1) (0.15.2)\n"
     ]
    }
   ],
   "source": [
    "!pip install reformer-pytorch==0.14\n",
    "!pip install transformers==2.4.1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 196,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/kaggle\n",
      "Cloning into 'reformer-chinese'...\n",
      "remote: Enumerating objects: 104, done.\u001b[K\n",
      "remote: Counting objects: 100% (104/104), done.\u001b[K\n",
      "remote: Compressing objects: 100% (71/71), done.\u001b[K\n",
      "remote: Total 104 (delta 62), reused 70 (delta 32), pack-reused 0\u001b[K\n",
      "Receiving objects: 100% (104/104), 140.65 KiB | 0 bytes/s, done.\n",
      "Resolving deltas: 100% (62/62), done.\n"
     ]
    }
   ],
   "source": [
    "%cd /kaggle\n",
    "!rm -rf reformer-chinese\n",
    "!git clone https://github.com/napoler/reformer-chinese.git"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 197,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/kaggle/reformer-chinese\n"
     ]
    }
   ],
   "source": [
    "%cd reformer-chinese"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 198,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-rw-r--r-- 1 root root 12678852 Mar  4 07:45 /kaggle/reformer-chinese/data/train.txt\n"
     ]
    }
   ],
   "source": [
    "!mkdir /kaggle/reformer-chinese/data/\n",
    "!shuf -n 3000 /kaggle/input/gpt2writedata/train.txt > /kaggle/reformer-chinese/data/train.txt\n",
    "!ls -l /kaggle/reformer-chinese/data/train.txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 199,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " [KW] 主人,疯起来,疯狗,还好,搜索,乱跑,狗狗,金毛,前段时间,致命 [/KW]  [TT] 尽量不要带出去的5种狗狗，一转眼可能就不见了，很容易弄丢 [/TT] [SM] 金毛就是这5种狗狗之一。谁叫它走，它都会特别乖巧的跟着那个人走。如果你带着它出去的话，你就会知道，疯狗这个词是怎么来的了。这种狗不管是在外面还是在家里，都是特别疯的。泰迪也是一种出门就会乱跑的够狗狗。 [/SM] [CONTNET] 小编前段时间养过一只狗，是一只金毛。本以为能够养它个七八年，最不济，也能给它养老送它走。结果没想到的是，它竟然在一次，我带着他散步的时候，弄丢了，再也找不回来了。小编当时特别难过，也很不服气，于是上网搜索了一下，发现，有5种狗是尽量别带出去的，因为带它们出去玩的话，一转眼就不见了，真的很容易会把它们给搞丢的。 [SEP]  [SEP] 第一种：金毛犬 [SEP]  [SEP] 金毛就是这5种狗狗之一。它虽然是很听话的狗狗吧，但是它也有一个致命的缺点，那就是它不认主人。谁叫它走，它都会特别乖巧的跟着那个人走。所以大家带着金毛出去玩的时候，一定要看好它了，牵引绳是一定要带的。 [SEP]  [SEP] 第二种：柴犬 [SEP]  [SEP] 说实话，柴犬它在家里表现的还是不错的，虽然偶尔也还是会发神经。但是它总体来说，还是很乖巧的。这是在你不带着它出去的前提下。如果你带着它出去的话，你就会知道，疯狗这个词是怎么来的了。它会四处乱跑，到最后，肯定就会跑丢了。 [SEP]  [SEP] 第三种：哈士奇 [SEP]  [SEP] 说起疯狗，怎么可能忘得了疯二哈？这种狗不管是在外面还是在家里，都是特别疯的。不过主人都知道，让它在外面疯还好一些，至少家具什么的还能够保住。可是带着它出去的话，可一定要看好它。它疯起来比柴犬疯多了。 [SEP]  [SEP] 第四种：萨摩耶 [SEP]  [SEP] 别的狗狗都是疯，都是乱跑，而萨摩耶不是，它只是有点傻而已。它根本不知道陌生人跟主人的区别。在它的心里，周围所有的人都是好人，都是它的朋友或者主人。这就代表着，它是很容易被人带走的。 [SEP]  [SEP] 第五种狗狗：泰迪犬 [SEP]  [SEP] 泰迪也是一种出门就会乱跑的够狗狗。所以大家在带着泰迪出去玩的时候，也是一定要看好它呦。因为说不定，它就找了个小媳妇，跟着它回家了。 [SEP]  [SEP]  [/CONTNET] [PT] 尽量不要带出去的5种狗狗，一转眼可能就不见了，很容易弄丢 [/PT] [END]\n",
      " [KW] 侵权,养殖羊驼,驮运,来源,作者,适合,养殖,羊驼养殖,删除,网络,中国养殖,图片,羊驼 [/KW]  [TT] 羊驼很可爱，为什么很少有人养？浅谈中国农村不养羊驼的原因 [/TT] [SM] 笔者第一次认识羊驼是在电视剧里面。羊驼虽然好养，但其实对圈舍的要求也很高的。成年的羊驼一年只能生产一次羊驼。所以，我国的农村对羊驼的认可率并不高。因此，羊驼养殖市场虽然在我国的市场还很空白，但并没有多少人敢大规模养殖。 [/SM] [CONTNET] 原创|纯天景色，图片来源|网咯，如有侵权，请联系作者删除，未经允许禁止任何形式的抄袭和转载。 [SEP]  [SEP] 对于现代的网络十大神兽，经常泡在网上的人都知道是什么。网络的十大神兽之中，有一种神兽，模样明明很可爱，但还是因为生活习性被冠以神兽的称号。而且一来就是十大神兽之首，这就是羊驼。 [SEP]  [SEP] 图片来源网络如有侵权请联系作者删除 [SEP]  [SEP] 羊驼是美洲印第安人驯化的驮运工具。成年的羊驼的驮运能力很强，现代养殖羊驼的主要目的却不再是驮运货物，而是羊驼的毛。和绵阳相比，羊驼的毛要更加珍贵，因为羊驼的养殖并不多，现在已知的羊驼养殖不过300多万只，主要生活在南美和澳大利亚。 [SEP]  [SEP] 图片来源网络如有侵权请联系作者删除 [SEP]  [SEP] 笔者第一次认识羊驼是在电视剧里面。有一部歌颂青春爱情的电视剧里面，有一个男配角饲养了一头羊驼。笔者通过影像资料了解到，羊驼长得很漂亮。很适合当作宠物。 [SEP]  [SEP] 图片来源网络如有侵权请联系作者删除 [SEP]  [SEP] 后来，笔者发现，中国养殖羊驼的人并不多，这是什么原因呢？于是笔者了解了羊驼的养殖方法和养殖技巧，终于明白了并不是所有的国外的东西都像克氏原螯虾一样适合中国养殖的。羊驼明显不适合中国大规模的养殖。 [SEP]  [SEP] 羊驼虽然在中国有很大的市场，但是羊驼的进口和养殖至今仍然保留在大的批发商和养殖户手中。最初，羊驼进入中国的时候，一只能达到十几万元的高价。近几年，羊驼的价格虽然有回落，但对于许多农户来说，依旧是高价。并且，羊驼在中国的主要受众都是精英人士，普罗大众很难接受数万元一只的萌宠，哪怕它很好养，食用的饲料也不多。 [SEP]  [SEP] 图片来源网络如有侵权请联系作者删除 [SEP]  [SEP] 羊驼的主要养殖地现在是在秘鲁和澳大利亚。羊驼是高寒地区的物种，羊毛能够长达80厘米，是一种高档的纺织品的原料。但是在中国，并没有与养殖羊驼配套的纺织设备和纺织物市场。所以，羊驼在中国的养殖市场并不算太大。 [SEP]  [SEP] 图片来源网络如有侵权请联系作者删除 [SEP]  [SEP] 想要的到完美的羊驼毛，需要对羊驼的活动范围和食用草料等环节严格把控。中国的养殖户很少有这样的精力去管理羊驼的养殖。这也是制约羊驼在中国生长扎根的原因。 [SEP]  [SEP] 图片来源网络如有侵权请联系作者删除 [SEP]  [SEP] 羊驼虽然好养，但其实对圈舍的要求也很高的。同绵羊相比，羊驼圈舍的建造成本，需要的草地都是苛刻的，同样是用来产毛的动物，羊驼一年只能剪毛一次。并且对青草的需求量也是巨大的。一亩无污染的草地至多只能养殖15只羊驼。而养羊可以是羊驼的一倍以上。 [SEP]  [SEP] 图片来源网络如有侵权请联系作者删除 [SEP]  [SEP] 养殖过牛羊的草地是不适合再养殖羊驼的。羊驼的毛极容易附着别的动物的杂毛，以及地上的异物。这样剪下里的羊毛由于掺杂异物，售价并不高。 [SEP]  [SEP] 图片来源网络如有侵权请联系作者删除 [SEP]  [SEP] 羊驼的繁殖率也不高。成年的羊驼一年只能生产一次羊驼。而且羊驼的胚胎死亡率也是比较高的。所以，我国的农村对羊驼的认可率并不高。 [SEP]  [SEP] 图片来源网络如有侵权请联系作者删除 [SEP]  [SEP] 现在，羊驼的主要作用是用于宠物养殖和展览。虽然，羊驼其实很适合我国的环境生长。但是，羊驼毕竟是外来物种，目前的受众依旧是小众，走红的原因也是因为网络，农村的养殖户对羊驼在我国的养殖前景还不是特别的确定。因此，羊驼养殖市场虽然在我国的市场还很空白，但并没有多少人敢大规模养殖。 [SEP]  [SEP] 图片来源网络如有侵权请联系作者删除 [SEP]  [SEP] 有的人会说，羊驼还能驮运。但是在我国，其实有比羊驼更适合的驮运动物，比如高原上的牦牛。草原上的骡马，平原里面的驴。因此，养殖羊驼做驮运并不现实。 [SEP]  [SEP] 图片来源网络如有侵权请联系作者删除 [SEP]  [SEP] 网络上有关于出租羊驼展览等多项增加羊驼身上附加值的办法，但其实都不适合农村的养殖户。养殖没前景，不能带来充分的附加价值，这也是农村很少有养殖羊驼的原因。 [SEP]  [SEP] 图片来源网络如有侵权请联系作者删除 [SEP]  [SEP] 对于中国的农民来说，养殖一种不清楚未来的新物种是一种极大的挑战。想要在中国推行养殖羊驼，替代绵羊在中国毛纺织品中的地位任重而道远。 [SEP]  [SEP] 图片来源网络如有侵权请联系作者删除 [SEP]  [SEP] 简单的概括一下，中国的农村为什么很少有养殖羊驼。一是养殖陈更高，二是养殖环境要求高，三是市场受众还不明确，四是农村养殖户很少有拼一把的勇气。这些原因，制约着中国羊驼养殖业的发展。 [SEP]  [SEP] 以上仅代表纯天景色个人观点，不代表任何组织和个人。 [SEP]  [SEP]  [/CONTNET] [PT] 羊驼很可爱，为什么很少有人养？浅谈中国农村不养羊驼的原因 [/PT] [END]\n",
      " [KW] 证书,看不到,宠物犬,看去,消费者,眼部,购买,品相,血统证书,血统,称为,笑容 [/KW]  [TT] 如何挑选一只好的比熊犬，要注意哪些方面，别被骗了 [/TT] [SM] 因为现在的狗贩子们都很会看市场，他们知道这种可爱，娇小的小型犬受到了很多人的喜欢。所以他们经常会拿一些别的狗来忽悠消费者，消费者们首先要看的就是这只比熊犬的整体外形，因为一般人都会选择买只幼犬。一只好的比熊犬，它们的身形一定要稳健扎实，虽然比熊犬很瘦，但是你摸上去隔着毛发也能摸到肉的那一种。如果它们能够出示出来这只比熊犬的血统证书，那么就证明这只比熊犬是一只好的品种，品相各方面都不会很差。如果没有证书，可能就是一般的宠物级比熊犬，这种比熊犬的品相也就是一般。 [/SM] [CONTNET] 现在养宠物的人们越来越多了，许多人都想挑选一只自己喜欢的宠物犬来陪着自己。但是现在市场上的狗贩子们很多，他们经常会以次充好，拿一些串狗，病狗甚至品相极差的狗，来忽悠那些不懂的消费者。以至于现在有许多的人都在挑选的方面操碎了心，大多人都想养一只品相好的宠物犬。比熊犬是一种聪明可爱的小型宠物犬，它有着一身毛茸茸的白色毛发，并且它们的毛发很胀很蓬松，使它们远远看去，就像一个球一样。它们的性格非常温顺，有些粘人，并且还不掉毛，非常适合在家饲养，受到了很多人的喜爱。那该如何挑选一只好的比熊犬呢，要注意哪些方面呢，小编跟你讲讲，别再被骗了。 [SEP]  [SEP] 因为现在的狗贩子们都很会看市场，他们知道这种可爱，娇小的小型犬受到了很多人的喜欢。所以他们经常会拿一些别的狗来忽悠消费者，消费者们首先要看的就是这只比熊犬的整体外形，因为一般人都会选择买只幼犬。一只好的比熊犬，它们的身形一定要稳健扎实，虽然比熊犬很瘦，但是你摸上去隔着毛发也能摸到肉的那一种。一只好的比熊犬的外貌不会很差，而且它们身体的各个部位的毛都要丰满，柔顺。它们的嘴不是方形的，头部要圆滑，并且整体都要均匀。 [SEP]  [SEP] 另外要看它们的精神面貌，一只好的比熊犬性格是调皮粘人的，它们不怕生，对于每个人它们都会好奇的扑上去和你玩耍。它们的眼神一定要清澈，清晰，看人的目光一定要温顺，对眼看去一般看不到白眼球为好。它们的嘴上总是挂着笑容，也被称为笑容最甜的狗狗。要仔细的观察比熊犬的眼部有没有眼屎，以及它们的鼻子上是否水润，一只好的比熊犬没有这些状况。 [SEP]  [SEP] 另外，如果想挑一只品相各方面都比较好的比熊犬，大家可以到正规的犬舍，宠物店去购买。如果它们能够出示出来这只比熊犬的血统证书，那么就证明这只比熊犬是一只好的品种，品相各方面都不会很差。如果没有证书，可能就是一般的宠物级比熊犬，这种比熊犬的品相也就是一般。所以各位家长在购买前要问一问，另外也可以看比熊犬的父母亲的血统是否优良，如果它的父母血统优良，并且都有血统证书，那么这只比熊犬的品相以及各方面都不会差，但是价格同样也会更高一些。 [SEP]  [SEP] 大家知道该如何挑选一只好的比熊犬了吗，小编祝愿大家能够挑选到一只自己满意的，健康的比熊犬。 [SEP]  [SEP]  [/CONTNET] [PT] 如何挑选一只好的比熊犬，要注意哪些方面，别被骗了 [/PT] [END]\n",
      " [KW] 集体,野熊,发现,冬眠,树洞,活动,民族,鄂伦春族猎人,猎犬,猎人,狩猎 [/KW]  [TT] 鄂伦春人原始狩猎所培养出的团结和勇敢精神，一代又一代的流传着 [/TT] [SM] 打围是鄂伦春族人集体狩猎的方式。早先，这种集体出猎是由鄂伦春族组织各家族成员进行。由于世代从事传统狩猎生产，并且从年龄很小的儿帝时期就开始接受各种狩猎本领的训练，使得这个民族的成年男子几乎个个都是好猎手。埋伏在左右的猎人们只待野熊受惊逃出树洞便猎枪齐发将它猎获。对于在“天仓”中冬眠的黑熊，猎人们则首先用以木棒敲击树干的方法“叫仓”，如若不能奏效，便用长长的木杆挑起汗味浓重的帽子或衣物诱使黑熊“出仓”。 [/SM] [CONTNET] 打围是鄂伦春族人集体狩猎的方式。早先，这种集体出猎是由鄂伦春族组织各家族成员进行。自枪支传入之后，这种集体打猎活动打破了家庭界限，由临时自发组成的“打围”小组“昂阿”取而代之。每到适宜时候，猎人们便互相走家串户，相约进山。措人们首先推举一位德高望重、经验丰富的“塔坦达”为“昂阿”的首领。一路上，“昂阿”的成员一切听从首领的指挥，砍柴、做饭、搭帐、喂马饲犬。 [SEP]  [SEP] 突然，一只猎犬发现了猎物的踪迹，只见“塔坦达”指挥若定，猎人们齐心协力，一会儿诱引，一会儿包抄围攻，一会儿又左堵右被。强悍的野兽终于被制服了，大家阵阵欢呼。猎期过后，所有捕获物在全体成员之同平均分配，而那位领头人“塔坦达”只是要了一份最次的兽肉，难怪猎人们对他如比敬重。 [SEP]  [SEP] 生活在东北地区的鄂伦春族是一个典型的森林民族，他们的生活与森林密不可分，狩猎是他们世代相传、赖以生存的本领。这个民族的先人，早在一千多年以前就以狩猎民族的身份见诸史书记载之中。由于世代从事传统狩猎生产，并且从年龄很小的儿帝时期就开始接受各种狩猎本领的训练，使得这个民族的成年男子几乎个个都是好猎手。鹿是鄂伦春人经常捕捉的猎物，但是，要论勇敢和惊心动魄，则要算鄂伦春族人捕猎野熊的场面。如果说猎鹿需要猎人灵活机敏的话，猎熊则更多的是需要猎人的勇敢，这是因为捕猎野熊要比猎鹿危险和困难得多。 [SEP]  [SEP] 在春季野熊刚刚结束冬眼和盛夏七月野熊的发情期，熊性会格外凶猛，有经验的鄂伦春族猎手在这两个时间都尽量避免猎熊。即使在其他季节，猎熊时也需要猎人沉着勇敢，有高超的枪法和机敏的反应能力，否则是很危险的。入秋以后，猎人可以循着野熊上树寻找野果时发出的声响发现目标。到了深秋野果落地，猎人则需借助猎犬寻找在柞树或榛树从中觅食的野熊。猎犬一且发现野熊，便会猛扑上去，与野熊展开搏斗。猎人则趁机迅速找好有利的地形，沉着瞄准，将野熊击倒。 [SEP]  [SEP] 相比之下，鄂伦春族猎人冬季打猎时进行的“掏仓”（猎熊），则在惊险中又有几分趣味。“仓”是指野熊为了冬眠而建造的窝，利用树洞或山洞建造的窝叫做“地仓”，在树上利用树权树枝建造的窝叫做“天仓”。野熊进窝开始冬眠时，大多身圆体肿，行动懒散笨重，反应灵敏程度会大大降低，于是猎人们趁此大好时机前来“拘仓”了。 [SEP]  [SEP] 他们踏着积雪在山林中搜索前进，猎犬一路跑在前面。突然，猎大发出信号，示意发现一处野熊冬眠的“地仓”。猎人们上前仔细观察，只见一株参天大树下一个深深的树洞洞口打开，看不清里面冬眠的是一只棕熊还是黑熊。猎人们迅速在四下埋伏停当，接着由一位经验丰富的猎人“叫仓”，以便把野熊轰赶出洞。只见他拾起大小石块使劲投向洞内。埋伏在左右的猎人们只待野熊受惊逃出树洞便猎枪齐发将它猎获。可是如果这只野熊就是不肯“出仓”，猎人就将一捆干草点燃，用长木杆挑入树洞。野熊终于“按捺不住”，冲出树洞，刚一露头就被猎枪击中。对于在“天仓”中冬眠的黑熊，猎人们则首先用以木棒敲击树干的方法“叫仓”，如若不能奏效，便用长长的木杆挑起汗味浓重的帽子或衣物诱使黑熊“出仓”。野熊被击倒之后，猎人们不是欢呼庆贺，而是虔诚地说它“睡着了”。 [SEP]  [SEP] 随后严格按照自己民族的信仰和禁忌处理猎获的野熊，使得惊险、有趣的猎熊活动又蒙上一层神秘的色彩。在提倡野生动物保护的今天，鄂伦春族猎人已经很少进行这种最能代表他们狩猎文化风貌的猎获野熊活动。但是他们在猎熊活动中培养起来的团结和勇敢机智的精神，仍然一代又一代的流传着。 [SEP]  [SEP]  [/CONTNET] [PT] 鄂伦春人原始狩猎所培养出的团结和勇敢精神，一代又一代的流传着 [/PT] [END]\n"
     ]
    }
   ],
   "source": [
    "!head -4 /kaggle/reformer-chinese/data/train.txt"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 训练"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 200,
   "metadata": {},
   "outputs": [],
   "source": [
    "# main()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 201,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mkdir: cannot create directory ‘/kaggle/working/model’: File exists\n"
     ]
    }
   ],
   "source": [
    "!mkdir /kaggle/working/model\n",
    "!cp /kaggle/input/reformerwritemodel/model/* /kaggle/working/model\n",
    "\n",
    "# !rm -rf /kaggle/working/model/*"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 修改为1.4版本\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "output_dir='/kaggle/working/model'\n",
    "model_path=os.path.join(output_dir, 'model.pt')\n",
    "# print(type(torch.load(model_path)))\n",
    "old=torch.load(model_path)\n",
    "new=torch.load(model_path)\n",
    "del new['net.net.pos_emb.emb.weight']\n",
    "new['net.net.pos_emb.weight']=old['net.net.pos_emb.emb.weight']\n",
    "# for it in new:\n",
    "#     print(it)\n",
    "del old\n",
    "torch.save(new,model_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 232,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "reading lines\n",
      "write json file success!\n",
      "args:\n",
      "Namespace(batch_size=30, bpe_token=False, device='cuda', epochs=2, fp16=False, fp16_opt_level='O1', gradient_accumulation=10, log_step=1, lr=0.0005, max_grad_norm=1.0, min_length=128, model_config='config/model_config_small.json', num_pieces=10, output_dir='/kaggle/working/model', pretrained_model='', raw=True, raw_data_path='data/train.json', segment=False, stride=768, tokenized_data_path='data/tokenized/', tokenizer_path='cache/vocab_small_terry_ai.txt', warmup_steps=2000)\n",
      "using device: cuda\n",
      "building files\n",
      "reading lines\n",
      "100%|███████████████████████████████████████████| 10/10 [01:03<00:00,  6.35s/it]\n",
      "finish\n",
      "files built\n",
      "calculating total steps\n",
      "100%|███████████████████████████████████████████| 10/10 [00:01<00:00,  6.29it/s]\n",
      "total steps = 36\n",
      "starting training\n",
      "epoch 1\n",
      "time: 2020-03-04 08:26:01.180997\n",
      "0it [00:00, ?it/s]epoch: 1  piece_num: 0 / 10  step: 10 / 36  loss: 1.3414510488510132\n",
      "1it [03:03, 183.58s/it]Traceback (most recent call last):\n",
      "  File \"train.py\", line 304, in <module>\n",
      "    main()\n",
      "  File \"train.py\", line 274, in main\n",
      "    loss = model(batch_inputs, return_loss = True)\n",
      "  File \"/opt/conda/lib/python3.6/site-packages/torch/nn/modules/module.py\", line 532, in __call__\n",
      "    result = self.forward(*input, **kwargs)\n",
      "  File \"/opt/conda/lib/python3.6/site-packages/reformer_pytorch/generative_tools.py\", line 80, in forward\n",
      "    out = self.net(xi, **kwargs)\n",
      "  File \"/opt/conda/lib/python3.6/site-packages/torch/nn/modules/module.py\", line 532, in __call__\n",
      "    result = self.forward(*input, **kwargs)\n",
      "  File \"/opt/conda/lib/python3.6/site-packages/reformer_pytorch/autopadder.py\", line 50, in forward\n",
      "    out = self.net(x, **kwargs)\n",
      "  File \"/opt/conda/lib/python3.6/site-packages/torch/nn/modules/module.py\", line 532, in __call__\n",
      "    result = self.forward(*input, **kwargs)\n",
      "  File \"/opt/conda/lib/python3.6/site-packages/reformer_pytorch/reformer_pytorch.py\", line 628, in forward\n",
      "    x = self.token_emb(x)\n",
      "  File \"/opt/conda/lib/python3.6/site-packages/torch/nn/modules/module.py\", line 532, in __call__\n",
      "    result = self.forward(*input, **kwargs)\n",
      "  File \"/opt/conda/lib/python3.6/site-packages/torch/nn/modules/sparse.py\", line 114, in forward\n",
      "    self.norm_type, self.scale_grad_by_freq, self.sparse)\n",
      "  File \"/opt/conda/lib/python3.6/site-packages/torch/nn/functional.py\", line 1484, in embedding\n",
      "    return torch.embedding(weight, input, padding_idx, scale_grad_by_freq, sparse)\n",
      "RuntimeError: Expected object of device type cuda but got device type cpu for argument #1 'self' in call to _th_index_select\n",
      "1it [03:03, 183.77s/it]\n"
     ]
    }
   ],
   "source": [
    "!python3 train.py --raw --epochs 2 --batch_size 30 --lr 0.0005 --gradient_accumulation 10 --output_dir /kaggle/working/model --num_pieces 10\n",
    "# !python3 train.py --raw --epochs 2 --batch_size 2 --lr 0.0005 --gradient_accumulation 2 --output_dir /kaggle/working/model --num_pieces 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 203,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import argparse\n",
    "# from reformer_pytorch import ReformerLM\n",
    "# from reformer_pytorch.generative_tools import TrainingWrapper\n",
    "# import torch\n",
    "# from transformers import *\n",
    "# import os\n",
    "# pretrained_weights = 'cache/vocab_small_terry_ai.txt'\n",
    "# device='cuda'\n",
    "# output_dir='/kaggle/working/model'\n",
    "\n",
    "# model_path=os.path.join(output_dir, 'model.pt')\n",
    "# tokenizer = BertTokenizer.from_pretrained(pretrained_weights)\n",
    "# model = ReformerLM(\n",
    "#     num_tokens= 13137,\n",
    "#     dim = 1024,\n",
    "#     depth = 12,\n",
    "#     max_seq_len = 4096,\n",
    "#     lsh_dropout = 0.1,\n",
    "#     causal = True,\n",
    "#     full_attn_thres = 1024\n",
    "# )\n",
    "\n",
    "# model_path=os.path.join(output_dir, 'model.pt')\n",
    "\n",
    "# if device=='cuda':\n",
    "#     model = TrainingWrapper(model, ignore_index = 0, pad_value = 0).cuda()\n",
    "# else:\n",
    "#     model = TrainingWrapper(model, ignore_index = 0, pad_value = 0)\n",
    "# # print(model)\n",
    "# for it in model.cpu().state_dict():\n",
    "#     print(it)\n",
    "\n",
    "# print('++++'*20)\n",
    "# if os.path.isfile(model_path):\n",
    "#     # if so, load them\n",
    "\n",
    "\n",
    "# #     \n",
    "#     print('++++'*20)\n",
    "#     model.load_state_dict(torch.load(model_path))\n",
    "    \n",
    "    \n",
    "# model_cpu_path=os.path.join(output_dir, 'model_cpu.pt')\n",
    "# torch.save(model.cpu().state_dict(), model_cpu_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 204,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Package                            Version             Location      \n",
      "---------------------------------- ------------------- --------------\n",
      "absl-py                            0.9.0               \n",
      "affine                             2.3.0               \n",
      "alabaster                          0.7.12              \n",
      "albumentations                     0.4.3               \n",
      "alembic                            1.4.0               \n",
      "allennlp                           0.9.0               \n",
      "altair                             4.0.1               \n",
      "anaconda-client                    1.7.2               \n",
      "anaconda-navigator                 1.9.7               \n",
      "anaconda-project                   0.8.3               \n",
      "annoy                              1.16.3              \n",
      "appdirs                            1.4.3               \n",
      "arrow                              0.15.5              \n",
      "arviz                              0.6.1               \n",
      "asn1crypto                         1.3.0               \n",
      "astor                              0.8.1               \n",
      "astroid                            2.3.3               \n",
      "astropy                            3.2.3               \n",
      "atomicwrites                       1.3.0               \n",
      "attrs                              19.3.0              \n",
      "audioread                          2.1.8               \n",
      "Babel                              2.8.0               \n",
      "backcall                           0.1.0               \n",
      "backports.functools-lru-cache      1.6.1               \n",
      "backports.os                       0.1.1               \n",
      "backports.shutil-get-terminal-size 1.0.0               \n",
      "backports.tempfile                 1.0                 \n",
      "backports.weakref                  1.0.post1           \n",
      "Baker                              1.3                 \n",
      "basemap                            1.2.0               \n",
      "bayesian-optimization              1.0.2               \n",
      "bayespy                            0.5.19              \n",
      "bcolz                              1.2.1               \n",
      "beautifulsoup4                     4.8.2               \n",
      "biopython                          1.76                \n",
      "bitarray                           1.2.1               \n",
      "bkcharts                           0.2                 \n",
      "bleach                             3.1.0               \n",
      "blis                               0.4.1               \n",
      "bokeh                              1.4.0               \n",
      "Boruta                             0.3                 \n",
      "boto                               2.49.0              \n",
      "boto3                              1.11.15             \n",
      "botocore                           1.14.15             \n",
      "Bottleneck                         1.3.1               \n",
      "bq-helper                          0.4.1               /src/bq-helper\n",
      "bqplot                             0.12.3              \n",
      "branca                             0.3.1               \n",
      "brewer2mpl                         1.4.1               \n",
      "cachetools                         4.0.0               \n",
      "cairocffi                          1.1.0               \n",
      "CairoSVG                           2.4.2               \n",
      "Cartopy                            0.17.0              \n",
      "catalogue                          1.0.0               \n",
      "catalyst                           20.2.1              \n",
      "catboost                           0.21                \n",
      "category-encoders                  2.1.0               \n",
      "certifi                            2019.11.28          \n",
      "cesium                             0.9.12              \n",
      "cffi                               1.14.0              \n",
      "cftime                             1.0.4.2             \n",
      "chainer                            7.1.0               \n",
      "chainer-chemistry                  0.7.0               \n",
      "chainercv                          0.13.1              \n",
      "chardet                            3.0.4               \n",
      "chart-studio                       1.0.0               \n",
      "cleverhans                         3.0.1               \n",
      "Click                              7.0                 \n",
      "click-plugins                      1.1.1               \n",
      "cliff                              2.18.0              \n",
      "cligj                              0.5.0               \n",
      "cloudpickle                        1.2.2               \n",
      "clyent                             1.2.2               \n",
      "cmd2                               0.8.9               \n",
      "cmudict                            0.4.4               \n",
      "colorama                           0.4.3               \n",
      "colorcet                           2.0.2               \n",
      "colorlog                           4.1.0               \n",
      "colorlover                         0.3.0               \n",
      "conda                              4.8.2               \n",
      "conda-build                        3.18.11             \n",
      "conda-package-handling             1.6.0               \n",
      "conda-verify                       3.4.2               \n",
      "ConfigArgParse                     1.0                 \n",
      "configparser                       4.0.2               \n",
      "confuse                            1.0.0               \n",
      "conllu                             1.3.1               \n",
      "contextily                         1.0rc2              \n",
      "contextlib2                        0.6.0.post1         \n",
      "convertdate                        2.2.0               \n",
      "conx                               3.7.10              \n",
      "coverage                           5.0.3               \n",
      "crc32c                             2.0                 \n",
      "cryptography                       2.3.1               \n",
      "cssselect2                         0.2.2               \n",
      "cufflinks                          0.17.0              \n",
      "cupy-cuda101                       7.1.1               \n",
      "CVXcanon                           0.1.1               \n",
      "cvxpy                              1.0.25              \n",
      "cycler                             0.10.0              \n",
      "cymem                              2.0.3               \n",
      "cysignals                          1.10.2              \n",
      "Cython                             0.29.15             \n",
      "cytoolz                            0.10.1              \n",
      "dask                               2.10.1              \n",
      "dask-glm                           0.2.0               \n",
      "dask-ml                            1.2.0               \n",
      "dask-xgboost                       0.1.10              \n",
      "dataclasses                        0.7                 \n",
      "datashader                         0.10.0              \n",
      "datashape                          0.5.2               \n",
      "deap                               1.3.1               \n",
      "decorator                          4.4.1               \n",
      "deepdish                           0.3.6               \n",
      "defusedxml                         0.6.0               \n",
      "Delorean                           1.0.0               \n",
      "Deprecated                         1.2.7               \n",
      "descartes                          1.1.0               \n",
      "dill                               0.3.1.1             \n",
      "dipy                               1.1.1               \n",
      "distributed                        2.10.0              \n",
      "docker-pycreds                     0.4.0               \n",
      "docopt                             0.6.2               \n",
      "docutils                           0.15.2              \n",
      "dora                               0.1                 \n",
      "earthengine-api                    0.1.213             \n",
      "ecos                               2.0.7.post1         \n",
      "editdistance                       0.5.3               \n",
      "edward                             1.3.5               \n",
      "eli5                               0.10.1              \n",
      "emoji                              0.5.4               \n",
      "en-core-web-lg                     2.2.5               \n",
      "en-core-web-sm                     2.2.5               \n",
      "entrypoints                        0.3                 \n",
      "essentia                           2.1b6.dev184        \n",
      "et-xmlfile                         1.0.1               \n",
      "fancyimpute                        0.5.4               \n",
      "fastai                             1.0.60              \n",
      "fastcache                          1.1.0               \n",
      "fasteners                          0.14.1              \n",
      "fastFM                             0.2.11              \n",
      "fastprogress                       0.2.2               \n",
      "fastrlock                          0.4                 \n",
      "fasttext                           0.9.1               \n",
      "fbpca                              1.0                 \n",
      "fbprophet                          0.5                 \n",
      "feather-format                     0.4.0               \n",
      "featuretools                       0.13.2              \n",
      "filelock                           3.0.12              \n",
      "Fiona                              1.8.13              \n",
      "fitter                             1.2.0               \n",
      "flake8                             3.7.9               \n",
      "flaky                              3.6.1               \n",
      "flashtext                          2.7                 \n",
      "Flask                              1.1.1               \n",
      "Flask-Cors                         3.0.8               \n",
      "folium                             0.10.1              \n",
      "fsspec                             0.6.2               \n",
      "ftfy                               5.6                 \n",
      "funcsigs                           1.0.2               \n",
      "funcy                              1.14                \n",
      "fury                               0.4.0               \n",
      "future                             0.18.2              \n",
      "fuzzywuzzy                         0.17.0              \n",
      "gast                               0.2.2               \n",
      "gatspy                             0.3                 \n",
      "gensim                             3.8.1               \n",
      "geographiclib                      1.50                \n",
      "Geohash                            1.0                 \n",
      "geojson                            2.5.0               \n",
      "geopandas                          0.6.3               \n",
      "geoplot                            0.4.0               \n",
      "geopy                              1.21.0              \n",
      "geoviews                           1.6.1               \n",
      "gevent                             1.4.0               \n",
      "ggplot                             0.11.5              \n",
      "gitdb2                             2.0.6               \n",
      "GitPython                          3.0.7               \n",
      "glmnet-py                          0.1.0b2             \n",
      "glob2                              0.7                 \n",
      "gluoncv                            0.6.0               \n",
      "gluonnlp                           0.8.3               \n",
      "gmpy2                              2.0.8               \n",
      "google-api-core                    1.16.0              \n",
      "google-api-python-client           1.7.11              \n",
      "google-auth                        1.11.0              \n",
      "google-auth-httplib2               0.0.3               \n",
      "google-auth-oauthlib               0.4.1               \n",
      "google-cloud-automl                0.10.0              \n",
      "google-cloud-bigquery              1.12.1              \n",
      "google-cloud-core                  1.3.0               \n",
      "google-cloud-storage               1.26.0              \n",
      "google-pasta                       0.1.8               \n",
      "google-resumable-media             0.5.0               \n",
      "googleapis-common-protos           1.51.0              \n",
      "gplearn                            0.4.1               \n",
      "gpxpy                              1.4.0               \n",
      "gql                                0.2.0               \n",
      "graphql-core                       1.1                 \n",
      "graphviz                           0.8.4               \n",
      "greenlet                           0.4.15              \n",
      "grpcio                             1.27.1              \n",
      "gym                                0.16.0              \n",
      "h2o                                3.28.0.3            \n",
      "h5py                               2.10.0              \n",
      "hallucinate                        0.0.1               \n",
      "haversine                          2.2.0               \n",
      "heamy                              0.0.7               \n",
      "HeapDict                           1.0.1               \n",
      "hep-ml                             0.6.0               \n",
      "hmmlearn                           0.2.3               \n",
      "holidays                           0.9.12              \n",
      "holoviews                          1.12.7              \n",
      "hpsklearn                          0.1.0               \n",
      "html5lib                           1.0.1               \n",
      "htmlmin                            0.1.12              \n",
      "httplib2                           0.17.0              \n",
      "httplib2shim                       0.0.3               \n",
      "humanize                           1.0.0               \n",
      "hunspell                           0.5.5               \n",
      "husl                               4.0.3               \n",
      "hyperopt                           0.2.3               \n",
      "hypertools                         0.6.2               \n",
      "hypothesis                         5.4.1               \n",
      "ibis-framework                     1.2.0               \n",
      "idna                               2.8                 \n",
      "imagecodecs-lite                   2019.12.3           \n",
      "ImageHash                          4.0                 \n",
      "imageio                            2.6.1               \n",
      "imagesize                          1.2.0               \n",
      "imbalanced-learn                   0.6.1               \n",
      "imgaug                             0.2.6               \n",
      "implicit                           0.4.2               \n",
      "importlib-metadata                 1.5.0               \n",
      "ipykernel                          5.1.1               \n",
      "ipython                            7.12.0              \n",
      "ipython-genutils                   0.2.0               \n",
      "ipywidgets                         7.5.1               \n",
      "iso3166                            1.0.1               \n",
      "isort                              4.3.21              \n",
      "isoweek                            1.3.3               \n",
      "itsdangerous                       1.1.0               \n",
      "Janome                             0.3.10              \n",
      "jax                                0.1.59              \n",
      "jaxlib                             0.1.38              \n",
      "jdcal                              1.4.1               \n",
      "jedi                               0.16.0              \n",
      "jeepney                            0.4.2               \n",
      "jieba                              0.42.1              \n",
      "Jinja2                             2.11.1              \n",
      "jmespath                           0.9.4               \n",
      "joblib                             0.14.1              \n",
      "json5                              0.9.1               \n",
      "jsonnet                            0.15.0              \n",
      "jsonpickle                         1.2                 \n",
      "jsonschema                         3.2.0               \n",
      "jupyter                            1.0.0               \n",
      "jupyter-client                     5.3.4               \n",
      "jupyter-console                    6.1.0               \n",
      "jupyter-core                       4.6.1               \n",
      "jupyterlab                         1.2.6               \n",
      "jupyterlab-server                  1.0.6               \n",
      "kaggle-environments                0.1.6               \n",
      "Keras                              2.3.1               \n",
      "Keras-Applications                 1.0.8               \n",
      "Keras-Preprocessing                1.1.0               \n",
      "keras-rcnn                         0.0.2               \n",
      "keras-resnet                       0.2.0               \n",
      "keras-rl                           0.4.2               \n",
      "keras-tqdm                         2.0.1               \n",
      "keyring                            21.1.0              \n",
      "kiwisolver                         1.1.0               \n",
      "kmapper                            1.2.0               \n",
      "kmeans-smote                       0.1.2               \n",
      "kmodes                             0.10.1              \n",
      "knnimpute                          0.1.0               \n",
      "langdetect                         1.0.7               \n",
      "langid                             1.1.6               \n",
      "Lasagne                            0.2.dev1            \n",
      "lazy-object-proxy                  1.4.3               \n",
      "learntools                         0.3.4               \n",
      "leven                              1.0.4               \n",
      "libarchive-c                       2.8                 \n",
      "librosa                            0.7.2               \n",
      "lief                               0.9.0               \n",
      "lightfm                            1.15                \n",
      "lightgbm                           2.3.1               \n",
      "lime                               0.1.1.37            \n",
      "line-profiler                      3.0.2               \n",
      "llvmlite                           0.31.0              \n",
      "lml                                0.0.9               \n",
      "locket                             0.2.0               \n",
      "lunardate                          0.2.0               \n",
      "lxml                               4.5.0               \n",
      "lz4                                2.1.2               \n",
      "Mako                               1.1.1               \n",
      "mapclassify                        2.2.0               \n",
      "marisa-trie                        0.7.5               \n",
      "Markdown                           3.2                 \n",
      "markovify                          0.8.0               \n",
      "MarkupSafe                         1.1.1               \n",
      "matplotlib                         3.0.3               \n",
      "matplotlib-venn                    0.11.5              \n",
      "mccabe                             0.6.1               \n",
      "memory-profiler                    0.57.0              \n",
      "mercantile                         1.1.2               \n",
      "missingno                          0.4.2               \n",
      "mistune                            0.8.4               \n",
      "mizani                             0.6.0               \n",
      "mkl-fft                            1.0.12              \n",
      "mkl-random                         1.0.2               \n",
      "mkl-service                        2.0.2               \n",
      "ml-metrics                         0.1.4               \n",
      "mlcrate                            0.2.0               \n",
      "mlens                              0.2.3               \n",
      "mlxtend                            0.17.1              \n",
      "mmh3                               2.5.1               \n",
      "mne                                0.19.2              \n",
      "mnist                              0.2.2               \n",
      "mock                               4.0.1               \n",
      "monotonic                          1.5                 \n",
      "more-itertools                     8.2.0               \n",
      "mpld3                              0.3                 \n",
      "mplleaflet                         0.0.5               \n",
      "mpmath                             1.1.0               \n",
      "msgpack                            0.6.1               \n",
      "msgpack-numpy                      0.4.4.3             \n",
      "multipledispatch                   0.6.0               \n",
      "multiprocess                       0.70.9              \n",
      "munch                              2.5.0               \n",
      "murmurhash                         1.0.2               \n",
      "mxnet-cu101                        1.5.1.post0         \n",
      "navigator-updater                  0.2.1               \n",
      "nbconvert                          5.6.1               \n",
      "nbformat                           5.0.4               \n",
      "nervananeon                        2.6.0               \n",
      "netCDF4                            1.5.3               \n",
      "networkx                           2.4                 \n",
      "nibabel                            3.0.1               \n",
      "nilearn                            0.6.1               \n",
      "nltk                               3.2.4               \n",
      "nolearn                            0.6.2.dev0          \n",
      "nose                               1.3.7               \n",
      "notebook                           5.5.0               \n",
      "numba                              0.48.0              \n",
      "numdifftools                       0.9.39              \n",
      "numexpr                            2.6.9               \n",
      "numpy                              1.18.1              \n",
      "numpydoc                           0.9.2               \n",
      "nvidia-ml-py3                      7.352.0             \n",
      "oauthlib                           3.1.0               \n",
      "odfpy                              1.4.1               \n",
      "olefile                            0.46                \n",
      "onnx                               1.6.0               \n",
      "opencv-python                      4.2.0.32            \n",
      "openpyxl                           3.0.3               \n",
      "opt-einsum                         3.1.0               \n",
      "optuna                             1.1.0               \n",
      "orderedmultidict                   1.0.1               \n",
      "ortools                            7.5.7466            \n",
      "osmnx                              0.10                \n",
      "osqp                               0.6.1               \n",
      "overrides                          2.8.0               \n",
      "packaging                          20.1                \n",
      "palettable                         3.3.0               \n",
      "pandas                             0.25.3              \n",
      "pandas-datareader                  0.8.1               \n",
      "pandas-profiling                   2.4.0               \n",
      "pandas-summary                     0.0.7               \n",
      "pandasql                           0.7.3               \n",
      "pandoc                             1.0.2               \n",
      "pandocfilters                      1.4.2               \n",
      "param                              1.9.3               \n",
      "paramnb                            2.0.4               \n",
      "parsimonious                       0.8.1               \n",
      "parso                              0.6.1               \n",
      "partd                              1.1.0               \n",
      "path                               13.1.0              \n",
      "path.py                            12.4.0              \n",
      "pathlib2                           2.3.5               \n",
      "pathos                             0.2.5               \n",
      "pathtools                          0.1.2               \n",
      "patsy                              0.5.1               \n",
      "pbr                                5.4.4               \n",
      "pdf2image                          1.12.0              \n",
      "PDPbox                             0.2.0+13.g73c6966   \n",
      "pep8                               1.7.1               \n",
      "pexpect                            4.8.0               \n",
      "phik                               0.9.9               \n",
      "pickleshare                        0.7.5               \n",
      "Pillow                             5.4.1               \n",
      "pip                                20.0.2              \n",
      "pkginfo                            1.5.0.1             \n",
      "plac                               0.9.6               \n",
      "plotly                             4.5.0               \n",
      "plotly-express                     0.4.1               \n",
      "plotnine                           0.4.0               \n",
      "pluggy                             0.13.1              \n",
      "ply                                3.11                \n",
      "polyglot                           16.7.4              \n",
      "portalocker                        1.5.2               \n",
      "posix-ipc                          1.0.4               \n",
      "pox                                0.2.7               \n",
      "ppca                               0.0.4               \n",
      "ppft                               1.6.6.1             \n",
      "preprocessing                      0.1.13              \n",
      "preshed                            3.0.2               \n",
      "prettytable                        0.7.2               \n",
      "progressbar                        2.5                 \n",
      "progressbar2                       3.47.0              \n",
      "prometheus-client                  0.7.1               \n",
      "promise                            2.3                 \n",
      "prompt-toolkit                     3.0.3               \n",
      "pronouncing                        0.2.0               \n",
      "protobuf                           3.11.3              \n",
      "psutil                             5.6.7               \n",
      "ptyprocess                         0.6.0               \n",
      "pudb                               2019.2              \n",
      "py                                 1.8.1               \n",
      "py-cpuinfo                         5.0.0               \n",
      "py-lz4framed                       0.14.0              \n",
      "py-stringmatching                  0.4.1               \n",
      "py-stringsimjoin                   0.3.1               \n",
      "pyahocorasick                      1.4.0               \n",
      "pyaml                              19.12.0             \n",
      "PyArabic                           0.6.6               \n",
      "pyarrow                            0.16.0              \n",
      "pyasn1                             0.4.8               \n",
      "pyasn1-modules                     0.2.8               \n",
      "PyAstronomy                        0.14.0              \n",
      "pybind11                           2.4.3               \n",
      "PyBrain                            0.3.3               \n",
      "pycairo                            1.19.0              \n",
      "pycodestyle                        2.5.0               \n",
      "pycosat                            0.6.3               \n",
      "pycountry                          19.8.18             \n",
      "pycparser                          2.19                \n",
      "pycrypto                           2.6.1               \n",
      "pyct                               0.4.6               \n",
      "pycuda                             2019.1.2            \n",
      "pycurl                             7.43.0.2            \n",
      "pydash                             4.7.6               \n",
      "pydicom                            1.4.1               \n",
      "pydot                              1.4.1               \n",
      "pyeconometrics                     1.0.2               \n",
      "pyemd                              0.5.1               \n",
      "pyexcel-io                         0.5.20              \n",
      "pyexcel-ods                        0.5.6               \n",
      "pyfasttext                         0.4.6               \n",
      "pyflakes                           2.1.1               \n",
      "pyflux                             0.4.15              \n",
      "pyglet                             1.4.10              \n",
      "Pygments                           2.5.2               \n",
      "pykalman                           0.9.5               \n",
      "pykoko                             0.1.8               \n",
      "pyLDAvis                           2.1.2               \n",
      "pylint                             2.4.4               \n",
      "pymagnitude                        0.1.120             \n",
      "pymc3                              3.8                 \n",
      "PyMeeus                            0.3.6               \n",
      "pymongo                            3.10.1              \n",
      "Pympler                            0.8                 \n",
      "pynvrtc                            9.2                 \n",
      "pyocr                              0.7.2               \n",
      "pyodbc                             4.0.0-unsupported   \n",
      "PyOpenGL                           3.1.5               \n",
      "pyOpenSSL                          19.0.0              \n",
      "pypandoc                           1.4                 \n",
      "pyparsing                          2.4.6               \n",
      "pyPdf                              1.13                \n",
      "pyperclip                          1.7.0               \n",
      "PyPrind                            2.11.2              \n",
      "pyproj                             2.4.2.post1         \n",
      "pyrsistent                         0.15.7              \n",
      "pysal                              2.1.0               \n",
      "pyshp                              2.1.0               \n",
      "PySocks                            1.7.1               \n",
      "pystan                             2.19.1.1            \n",
      "pytagcloud                         0.3.5               \n",
      "pytesseract                        0.3.2               \n",
      "pytest                             5.0.1               \n",
      "pytest-arraydiff                   0.3                 \n",
      "pytest-astropy                     0.7.0               \n",
      "pytest-astropy-header              0.1.2               \n",
      "pytest-cov                         2.8.1               \n",
      "pytest-doctestplus                 0.5.0               \n",
      "pytest-mock                        2.0.0               \n",
      "pytest-openfiles                   0.4.0               \n",
      "pytest-pylint                      0.15.0              \n",
      "pytest-remotedata                  0.3.2               \n",
      "pytext-nlp                         0.1.2               \n",
      "python-dateutil                    2.8.1               \n",
      "python-editor                      1.0.4               \n",
      "python-igraph                      0.7.1.post7         \n",
      "python-Levenshtein                 0.12.0              \n",
      "python-louvain                     0.13                \n",
      "python-utils                       2.3.0               \n",
      "pytools                            2020.1              \n",
      "pytorch-ignite                     0.3.0               \n",
      "pytorch-pretrained-bert            0.6.2               \n",
      "pytorch-transformers               1.1.0               \n",
      "pytz                               2019.3              \n",
      "PyUpSet                            0.1.1.post7         \n",
      "pyviz-comms                        0.7.3               \n",
      "PyWavelets                         1.1.1               \n",
      "PyYAML                             5.3                 \n",
      "pyzmq                              18.1.1              \n",
      "qgrid                              1.2.0               \n",
      "QtAwesome                          0.6.1               \n",
      "qtconsole                          4.6.0               \n",
      "QtPy                               1.9.0               \n",
      "raccoon                            3.0.0               \n",
      "randomgen                          1.16.6              \n",
      "rasterio                           1.1.2               \n",
      "ray                                0.8.1               \n",
      "redis                              3.4.1               \n",
      "reformer-pytorch                   0.15.2              \n",
      "regex                              2020.1.8            \n",
      "requests                           2.22.0              \n",
      "requests-oauthlib                  1.3.0               \n",
      "resampy                            0.2.2               \n",
      "responses                          0.10.9              \n",
      "retrying                           1.3.3               \n",
      "revrand                            0.6.5               \n",
      "rf-perm-feat-import                0.1                 \n",
      "rgf-python                         3.7.0               \n",
      "rope                               0.16.0              \n",
      "rsa                                4.0                 \n",
      "Rtree                              0.8.3               \n",
      "ruamel-yaml                        0.15.87             \n",
      "s2sphere                           0.2.5               \n",
      "s3fs                               0.4.0               \n",
      "s3transfer                         0.3.3               \n",
      "sacred                             0.8.1               \n",
      "sacremoses                         0.0.38              \n",
      "safitty                            1.3                 \n",
      "scattertext                        0.0.2.58            \n",
      "scikit-image                       0.16.2              \n",
      "scikit-learn                       0.22.1              \n",
      "scikit-multilearn                  0.2.0               \n",
      "scikit-optimize                    0.7.2               \n",
      "scikit-plot                        0.3.7               \n",
      "scikit-surprise                    1.1.0               \n",
      "scipy                              1.4.1               \n",
      "scs                                2.1.1.post2         \n",
      "seaborn                            0.10.0              \n",
      "SecretStorage                      3.1.2               \n",
      "Send2Trash                         1.5.0               \n",
      "sentencepiece                      0.1.85              \n",
      "sentry-sdk                         0.14.1              \n",
      "setuptools                         45.2.0.post20200210 \n",
      "setuptools-git                     1.2                 \n",
      "shap                               0.34.0              \n",
      "Shapely                            1.7.0               \n",
      "shortuuid                          0.5.0               \n",
      "simplegeneric                      0.8.1               \n",
      "SimpleITK                          1.2.4               \n",
      "singledispatch                     3.4.0.3             \n",
      "six                                1.14.0              \n",
      "sklearn                            0.0                 \n",
      "sklearn-contrib-lightning          0.5.0               \n",
      "sklearn-contrib-py-earth           0.1.0+1.gdde5f89    \n",
      "sklearn-pandas                     1.8.0               \n",
      "smart-open                         1.9.0               \n",
      "smhasher                           0.150.1             \n",
      "smmap2                             2.0.5               \n",
      "snowballstemmer                    2.0.0               \n",
      "snuggs                             1.4.7               \n",
      "sortedcollections                  1.1.2               \n",
      "sortedcontainers                   2.1.0               \n",
      "SoundFile                          0.10.3.post1        \n",
      "soupsieve                          1.9.5               \n",
      "spacy                              2.2.3               \n",
      "spectral                           0.20                \n",
      "speedml                            0.9.3               \n",
      "Sphinx                             2.4.0               \n",
      "sphinx-rtd-theme                   0.2.4               \n",
      "sphinxcontrib-applehelp            1.0.1               \n",
      "sphinxcontrib-devhelp              1.0.1               \n",
      "sphinxcontrib-htmlhelp             1.0.2               \n",
      "sphinxcontrib-jsmath               1.0.1               \n",
      "sphinxcontrib-qthelp               1.0.2               \n",
      "sphinxcontrib-serializinghtml      1.1.3               \n",
      "sphinxcontrib-websupport           1.2.0               \n",
      "spyder                             3.3.6               \n",
      "spyder-kernels                     0.5.2               \n",
      "SQLAlchemy                         1.3.13              \n",
      "sqlparse                           0.3.0               \n",
      "squarify                           0.4.3               \n",
      "srsly                              1.0.1               \n",
      "statsmodels                        0.11.0              \n",
      "stemming                           1.0.1               \n",
      "stevedore                          1.32.0              \n",
      "stop-words                         2018.7.23           \n",
      "stopit                             1.1.2               \n",
      "subprocess32                       3.5.4               \n",
      "svgwrite                           1.3.1               \n",
      "sympy                              1.5.1               \n",
      "tables                             3.5.1               \n",
      "tabulate                           0.8.6               \n",
      "tblib                              1.6.0               \n",
      "tensorboard                        2.1.0               \n",
      "tensorboardX                       2.0                 \n",
      "tensorflow                         2.1.0               \n",
      "tensorflow-estimator               2.1.0               \n",
      "tensorflow-hub                     0.7.0               \n",
      "tensorflow-probability             0.9.0               \n",
      "Tensorforce                        0.5.3               \n",
      "tensorpack                         0.9.8               \n",
      "termcolor                          1.1.0               \n",
      "terminado                          0.8.3               \n",
      "terminalplot                       0.3.0               \n",
      "testpath                           0.4.4               \n",
      "textblob                           0.15.3              \n",
      "tflearn                            0.3.2               \n",
      "Theano                             1.0.4+34.g473d74ea4 \n",
      "thinc                              7.3.1               \n",
      "tifffile                           2019.7.26.2         \n",
      "tinycss2                           1.0.2               \n",
      "tokenizers                         0.0.11              \n",
      "toolz                              0.10.0              \n",
      "torch                              1.4.0               \n",
      "torchaudio                         0.4.0a0+719bcc7     \n",
      "torchtext                          0.5.0               \n",
      "torchvision                        0.5.0               \n",
      "tornado                            5.0.2               \n",
      "TPOT                               0.11.1              \n",
      "tqdm                               4.42.1              \n",
      "trackml                            0.1.12              \n",
      "traitlets                          4.3.3               \n",
      "traittypes                         0.2.1               \n",
      "transformers                       2.4.1               \n",
      "trueskill                          0.4.5               \n",
      "tsfresh                            0.14.1              \n",
      "typed-ast                          1.4.1               \n",
      "typing                             3.6.4               \n",
      "typing-extensions                  3.7.4.1             \n",
      "tzlocal                            2.0.0               \n",
      "umap-learn                         0.3.10              \n",
      "unicodecsv                         0.14.1              \n",
      "Unidecode                          1.1.1               \n",
      "update-checker                     0.16                \n",
      "uritemplate                        3.0.1               \n",
      "urllib3                            1.25.8              \n",
      "urwid                              2.1.0               \n",
      "vecstack                           0.4.0               \n",
      "vega3                              0.13.0              \n",
      "vida                               0.3                 \n",
      "visvis                             1.12.2              \n",
      "vowpalwabbit                       8.5.0               \n",
      "vtk                                8.1.2               \n",
      "Wand                               0.5.3               \n",
      "wandb                              0.8.27              \n",
      "wasabi                             0.6.0               \n",
      "watchdog                           0.10.2              \n",
      "wavio                              0.0.4               \n",
      "wcwidth                            0.1.8               \n",
      "webencodings                       0.5.1               \n",
      "websocket-client                   0.57.0              \n",
      "Werkzeug                           1.0.0               \n",
      "wfdb                               2.2.1               \n",
      "wheel                              0.34.2              \n",
      "widgetsnbextension                 3.5.1               \n",
      "word2number                        1.1                 \n",
      "Wordbatch                          1.4.4               \n",
      "wordcloud                          1.6.0               \n",
      "wordsegment                        1.3.1               \n",
      "wrapt                              1.11.2              \n",
      "wurlitzer                          2.0.0               \n",
      "xarray                             0.15.0              \n",
      "xgboost                            0.90                \n",
      "xlrd                               1.2.0               \n",
      "XlsxWriter                         1.2.7               \n",
      "xlwt                               1.3.0               \n",
      "xvfbwrapper                        0.2.9               \n",
      "xxhash                             1.3.0               \n",
      "yellowbrick                        1.0.1               \n",
      "zict                               1.0.0               \n",
      "zipp                               2.2.0               \n"
     ]
    }
   ],
   "source": [
    "!pip list\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 245,
   "metadata": {},
   "outputs": [
    {
     "ename": "KeyError",
     "evalue": "'net.net.pos_emb.emb.weight'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyError\u001b[0m                                  Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-245-9f3964d91adb>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0mold\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mload\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel_path\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      6\u001b[0m \u001b[0mnew\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mload\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel_path\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 7\u001b[0;31m \u001b[0;32mdel\u001b[0m \u001b[0mnew\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'net.net.pos_emb.emb.weight'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      8\u001b[0m \u001b[0mnew\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'net.net.pos_emb.weight'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mold\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'net.net.pos_emb.emb.weight'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      9\u001b[0m \u001b[0;31m# for it in new:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyError\u001b[0m: 'net.net.pos_emb.emb.weight'"
     ]
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 206,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1.4.0\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "print(torch.__version__)  #注意是双下划线"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 207,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import argparse\n",
    "# from reformer_pytorch import ReformerLM\n",
    "# from reformer_pytorch.generative_tools import TrainingWrapper\n",
    "# import torch\n",
    "# from transformers import *\n",
    "# import os\n",
    "# pretrained_weights = 'cache/vocab_small_terry_ai.txt'\n",
    "# device='cpu'\n",
    "# output_dir='/kaggle/working/model'\n",
    "\n",
    "\n",
    "# tokenizer = BertTokenizer.from_pretrained(pretrained_weights)\n",
    "# model = ReformerLM(\n",
    "#     num_tokens= 13137,\n",
    "#     dim = 1024,\n",
    "#     depth = 12,\n",
    "#     max_seq_len = 4096,\n",
    "#     lsh_dropout = 0.1,\n",
    "#     causal = True,\n",
    "#     full_attn_thres = 1024\n",
    "# )\n",
    "\n",
    "# model_path=os.path.join(output_dir, 'model.pt')\n",
    "\n",
    "# if device=='cuda':\n",
    "#     model = TrainingWrapper(model, ignore_index = 0, pad_value = 0).cuda()\n",
    "# else:\n",
    "#     model = TrainingWrapper(model, ignore_index = 0, pad_value = 0)\n",
    "# print(model)\n",
    "# if os.path.isfile(model_path):\n",
    "#     # if so, load them\n",
    "#     model.load_state_dict(torch.load(model_path))\n",
    "# print(model)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 生成内容"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 208,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "# from reformer_pytorch import ReformerLM\n",
    "# from reformer_pytorch.generative_tools import TrainingWrapper\n",
    "# import torch\n",
    "# from transformers import *\n",
    "\n",
    "# pretrained_weights = 'cache/vocab_small_terry_ai.txt'\n",
    "# device='cuda'\n",
    "# output_dir='/kaggle/working/model'\n",
    "\n",
    "# with torch.no_grad():\n",
    "#     tokenizer = BertTokenizer.from_pretrained(pretrained_weights)\n",
    "#     model = ReformerLM(\n",
    "#         num_tokens= 13137,\n",
    "#         dim = 1024,\n",
    "#         depth = 12,\n",
    "#         max_seq_len = 4096,\n",
    "#         lsh_dropout = 0.1,\n",
    "#         causal = True,\n",
    "#         full_attn_thres = 1024\n",
    "#     )\n",
    "\n",
    "#     model_path=os.path.join(output_dir, 'model.pt')\n",
    "\n",
    "#     if device=='cuda':\n",
    "#         model = TrainingWrapper(model, ignore_index = 0, pad_value = 0).cuda()\n",
    "#     else:\n",
    "#         model = TrainingWrapper(model, ignore_index = 0, pad_value = 0)\n",
    "\n",
    "#     if os.path.isfile(model_path):\n",
    "#         # if so, load them\n",
    "#         model.load_state_dict(torch.load(model_path))\n",
    "\n",
    "#     # sentence_0 = \"你是谁啊\"\n",
    "#     def auto_encode(sentence_0):\n",
    "#       # sentence_1 = \"我是谁啊\"\n",
    "#       sentence_1=None\n",
    "#       inputs_1 = tokenizer.encode_plus(sentence_0, sentence_1, add_special_tokens=True, return_tensors='pt')\n",
    "#       return inputs_1['input_ids']\n",
    "\n",
    "\n",
    "\n",
    "#     def get(start_text):\n",
    "#       \"\"\"\n",
    "#       获取预测文本\n",
    "#       \"\"\"\n",
    "#       # start_text=x_train_text[0][:5]\n",
    "#       initial =auto_encode(start_text).cuda()\n",
    "#       sample = model.generate(initial, 30, temperature=1., filter_thres = 0.9, eos_token = 1) # assume end token is 1, or omit and it will sample up to 100\n",
    "#       # print(sample)\n",
    "#       # print(sample.shape) # (1, <=100) token ids\n",
    "#       text = tokenizer.convert_ids_to_tokens(sample.tolist()[0])\n",
    "#       return text\n",
    "#     start_text=\"狗\"\n",
    "#     \"\".join(get(start_text))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 209,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# model.cpu()\n",
    "# torch.cuda.empty_cache()\n",
    "# del model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 210,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "RvtIXiTaNdR6"
   },
   "outputs": [],
   "source": [
    "# # should fit in ~ 5gb - 8k tokens\n",
    "\n",
    "# import torch\n",
    "# from reformer_pytorch import ReformerLM\n",
    "\n",
    "# model = ReformerLM(\n",
    "#     num_tokens= 20000,\n",
    "#     dim = 1024,\n",
    "#     depth = 12,\n",
    "#     max_seq_len = 8192,\n",
    "#     heads = 8,\n",
    "#     lsh_dropout = 0.1,\n",
    "#     layer_dropout = 0.1,  # layer dropout from 'Reducing Transformer Depth on Demand' paper\n",
    "#     emb_dim = 128,        # embedding factorization for further memory savings\n",
    "#     causal = True,        # auto-regressive or not\n",
    "#     bucket_size = 64,     # average size of qk per bucket, 64 was recommended in paper\n",
    "#     n_hashes = 4,         # 4 is permissible per author, 8 is the best but slower\n",
    "#     ff_chunks = 200,      # number of chunks for feedforward layer, make higher if there are memory issues\n",
    "#     weight_tie = False,   # tie parameters of each layer for no memory per additional depth\n",
    "#     attn_chunks = 8,        # process lsh attention in chunks, only way for memory to fit when scaling to 16k tokens\n",
    "#     num_mem_kv = 128,       # persistent learned memory key values, from all-attention paper\n",
    "#     twin_attention = False, # both branches of the reversible network will be attention\n",
    "#     use_full_attn = False,  # use full self attention, for comparison\n",
    "#     full_attn_thres = 1024, # use full attention if context length is less than set value\n",
    "#     use_scale_norm = False,  # use scale norm from 'Transformers without tears' paper\n",
    "#     one_value_head = False   # use one set of values for all heads from 'One Write-Head Is All You Need'\n",
    "# ).cuda()\n",
    "\n",
    "# x = torch.randint(0, 20000, (1, 8192)).long().cuda()\n",
    "# y = model(x) # (1, 8192, 20000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 211,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "8aJIZRKFNs_Q"
   },
   "outputs": [],
   "source": [
    "# x\n",
    "# y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "BXKUTSUHQ1Q-"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 212,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 82
    },
    "colab_type": "code",
    "id": "VyAqIL-3Q_xZ",
    "outputId": "a9b1ade6-e64b-4919-e626-9e2a34952aee"
   },
   "outputs": [],
   "source": [
    "import torch\n",
    "from transformers import *\n",
    "\n",
    "pretrained_weights = 'bert-base-chinese'\n",
    "tokenizer = BertTokenizer.from_pretrained(pretrained_weights)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 213,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "TOomxjFJRY9b"
   },
   "outputs": [],
   "source": [
    "# # sentence_0 = \"你是谁啊\"\n",
    "# def auto(sentence_0):\n",
    "#   # sentence_1 = \"我是谁啊\"\n",
    "#   sentence_1=None\n",
    "#   inputs_1 = tokenizer.encode_plus(sentence_0, sentence_1, add_special_tokens=True, return_tensors='pt')\n",
    "#   return inputs_1['input_ids']\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 214,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "Ue3i0MdWQCxT"
   },
   "outputs": [],
   "source": [
    "# import torch\n",
    "# import os\n",
    "# from torch import randint\n",
    "# import torch.nn as nn\n",
    "# from reformer_pytorch import ReformerLM\n",
    "# from reformer_pytorch.generative_tools import TrainingWrapper\n",
    "# device = 'cuda' if torch.cuda.is_available() else 'cpu'\n",
    "\n",
    "# # model = ReformerLM(\n",
    "# #     num_tokens= 20000,\n",
    "# #     dim = 1024,\n",
    "# #     depth = 12,\n",
    "# #     max_seq_len = 4096,\n",
    "# #     lsh_dropout = 0.1,\n",
    "# #     causal = True,\n",
    "# #     full_attn_thres = 1024\n",
    "# # )\n",
    "# model = ReformerLM(\n",
    "#     num_tokens= 20000,\n",
    "#     dim = 2048,\n",
    "#     depth = 12,\n",
    "#     max_seq_len = 4096,\n",
    "#     lsh_dropout = 0.1,\n",
    "#     causal = True,\n",
    "#     full_attn_thres = 2048\n",
    "# )\n",
    "\n",
    "# # 0 is used for padding and no loss to be calculated on it\n",
    "# model = TrainingWrapper(model, ignore_index = 0, pad_value = 0).to(device)\n",
    "\n",
    "# # the wrapper can handle evenly packed sequences\n",
    "# # x_train = randint(0, 20000, (3, 357))\n",
    "# # print(x_train)\n",
    "# # # print(len(x_train[1]))\n",
    "# # # # or if you have a list of uneven sequences, it will be padded for you\n",
    "# # x_train = [\n",
    "# #     randint(0, 20000, (120,)),\n",
    "# #     randint(0, 20000, (253,)),\n",
    "# #     randint(0, 20000, (846,))\n",
    "# # ]\n",
    "# # print(x_train)\n",
    "\n",
    "# x_train_text = [\n",
    "#     \"先说说狗是怎么来的，狗是由狼来的。但是它不是被人类驯化的，而是反过来，狼其中一部分主动驯化了人，它变成了狗。\",\n",
    "#     '我们都看过一本书《狼图腾》。这本书里讲狼是没有办法被驯化的，小狼你拿铁链给它锁上，它越长大，它的凶性，野性就越强悍，绝不屈服。所以狼是不可能被驯化的。那狼怎么就变成了狗呢？',\n",
    "#     '是这样一个过程，在北方远古的荒原上，有两支物种，这两支物种是世仇，它们不是天敌，因为他们互相之间没有兴趣。但是，它们之间互相争抢自然资源，这就是人类和狼。有限的猎物本来到了冬天就少，这两支队伍就抢来抢去。'\n",
    "# ]\n",
    "# x_train=[]\n",
    "# for it in x_train_text:\n",
    "#   x_train.append(auto(it)[0].to(device))\n",
    "# # print(x_train)\n",
    "# # # when training, set return_loss equal to True\n",
    "# model.train()\n",
    "\n",
    "\n",
    "# # scheduler = WarmupLinearSchedule(optimizer, warmup_steps=10,\n",
    "# #                                                       t_total=100)\n",
    "\n",
    "\n",
    "# num_train_epochs=30\n",
    "# weight_decay=0.0\n",
    "# learning_rate=5e-5\n",
    "# adam_epsilon=1e-8\n",
    "# warmup_steps=0\n",
    "# max_grad_norm=1.0\n",
    "# max_steps=-1\n",
    "# gradient_accumulation_steps=1\n",
    "# logging_steps=1000\n",
    "# save_steps=10000\n",
    "# no_decay = ['bias', 'LayerNorm.weight']\n",
    "# optimizer_grouped_parameters = [\n",
    "#     {\n",
    "#         'params': [p for n, p in model.named_parameters() if not any(nd in n for nd in no_decay)],\n",
    "#         'weight_decay': weight_decay\n",
    "#     },\n",
    "#     {\n",
    "#         'params': [p for n, p in model.named_parameters() if any(nd in n for nd in no_decay)],\n",
    "#         'weight_decay': 0.0\n",
    "#     }\n",
    "# ]\n",
    "\n",
    "\n",
    "\n",
    "# t_total = len(x_train_text)/gradient_accumulation_steps * num_train_epochs\n",
    "# # t_total=3/1*3\n",
    "# optimizer = AdamW(model.parameters(), lr=0.001, correct_bias=True)\n",
    "# optimizer = AdamW(optimizer_grouped_parameters, lr=learning_rate, eps=adam_epsilon)\n",
    "# scheduler = get_linear_schedule_with_warmup( optimizer=optimizer, num_warmup_steps=warmup_steps,num_training_steps=t_total)\n",
    "\n",
    "# # # checking if another optimizer/scheduler exists\n",
    "# if os.path.isfile('optimizer.pt') and os.path.isfile('scheduler.pt'):\n",
    "#     # if so, load them\n",
    "#     optimizer.load_state_dict(torch.load('optimizer.pt'))\n",
    "#     scheduler.load_state_dict(torch.load('scheduler.pt'))\n",
    "# if os.path.isfile('model.pt'):\n",
    "#     # if so, load them\n",
    "#     model.load_state_dict(torch.load('model.pt'))\n",
    "# loss_fn=nn.CrossEntropyLoss()\n",
    "\n",
    "# for i in range(num_train_epochs):\n",
    "#   loss = model(x_train, return_loss = True)\n",
    "#   # outputs = model(x_train)\n",
    "#   # outputs = torch.argmax(outputs, dim=-1)\n",
    "#   # loss = loss_fn(outputs.float(), labels.long())\n",
    "#   # loss.requires_grad = True\n",
    "\n",
    "#   loss.backward()\n",
    "#   optimizer.step()\n",
    "#   # scheduler.step()\n",
    "#   model.zero_grad()\n",
    "#   print(\"loss\",loss.item())\n",
    "\n",
    "# # # when evaluating, just use the generate function, which will default to top_k sampling with temperature of 1.\n",
    "# # initial = torch.tensor([[0]]).long() # assume 0 is start token\n",
    "# # sample = model.generate(initial, 100, temperature=1., filter_thres = 0.9, eos_token = 1) # assume end token is 1, or omit and it will sample up to 100\n",
    "# # print(sample.shape) # (1, <=100) token ids"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 215,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "ou1mZb17giLb"
   },
   "outputs": [],
   "source": [
    "# model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "exELoSY7jBnT"
   },
   "source": [
    "#保存模型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 216,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "p6ZDKkmrgEI7"
   },
   "outputs": [],
   "source": [
    "# import os\n",
    "# output_dir='./'\n",
    "\n",
    "# # torch.save(model, os.path.join(output_dir, 'model.pt'))\n",
    "# torch.save(model.state_dict(),  'model.pt')\n",
    "# # torch.save(optimizer.state_dict(), os.path.join(output_dir, 'optimizer.pt'))\n",
    "# # torch.save(scheduler.state_dict(), os.path.join(output_dir, 'scheduler.pt'))\n",
    "# torch.save(optimizer.state_dict(), 'optimizer.pt')\n",
    "# torch.save(scheduler.state_dict(),  'scheduler.pt')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "zmZs1xRfjD_5"
   },
   "source": [
    "#预测"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 217,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "tSKbV2PGlLd8"
   },
   "outputs": [],
   "source": [
    "# def get(start_text):\n",
    "#   \"\"\"\n",
    "#   获取预测文本\n",
    "#   \"\"\"\n",
    "#   # start_text=x_train_text[0][:5]\n",
    "#   initial =auto(start_text)\n",
    "#   sample = model.generate(initial, 30, temperature=1., filter_thres = 0.9, eos_token = 1) # assume end token is 1, or omit and it will sample up to 100\n",
    "#   # print(sample)\n",
    "#   # print(sample.shape) # (1, <=100) token ids\n",
    "#   text = tokenizer.convert_ids_to_tokens(sample.tolist()[0])\n",
    "#   return text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 218,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 629
    },
    "colab_type": "code",
    "id": "bqVSeFzflfSx",
    "outputId": "5c3de65a-146a-4e7e-ec8e-97ea7201f4f3"
   },
   "outputs": [],
   "source": [
    "# get(\"狗\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 219,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 89
    },
    "colab_type": "code",
    "id": "79ntWhksVE_3",
    "outputId": "5d925115-1906-4e66-822d-d6e1608e12c1"
   },
   "outputs": [],
   "source": [
    "# # initial = torch.tensor([[1044]]).long() # assume 0 is start token\n",
    "# start_text=x_train_text[0][:5]\n",
    "# initial =auto(start_text)\n",
    "# sample = model.generate(initial, 30, temperature=1., filter_thres = 0.9, eos_token = 1) # assume end token is 1, or omit and it will sample up to 100\n",
    "# print(sample)\n",
    "# print(sample.shape) # (1, <=100) token ids"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 220,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 53
    },
    "colab_type": "code",
    "id": "ZxPCskwmW_6F",
    "outputId": "c17bf28a-5ea4-48f8-b78a-bb63bf40c474"
   },
   "outputs": [],
   "source": [
    "# text = tokenizer.convert_ids_to_tokens(sample.tolist()[0])\n",
    "# print(x_train_text[0])\n",
    "# x_train_text[0][:5]+\"\".join(text)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "uKvdF0KcQFqA"
   },
   "source": []
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "name": "reformer_pytorch 内容创建系统.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
